{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3.7.3 64-bit ('.venv': venv)",
      "language": "python",
      "name": "python37364bitvenvvenv7b03464ca52642a09f5ea3f892125398"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3-final"
    },
    "toc": {
      "nav_menu": {},
      "number_sections": true,
      "sideBar": true,
      "skip_h1_title": false,
      "title_cell": "Table of Contents",
      "title_sidebar": "Contents",
      "toc_cell": false,
      "toc_position": {},
      "toc_section_display": true,
      "toc_window_display": false
    },
    "colab": {
      "name": "mlp-batchnorm.ipynb",
      "provenance": []
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a46b1e088aed4b04af22f27a0dfd431d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3f29e77055514bfeb063b608d530f5c5",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f8a8e08b1e6b410b9a182595c69f7921",
              "IPY_MODEL_6600e51e4e5c40909206459f2349fa4e"
            ]
          }
        },
        "3f29e77055514bfeb063b608d530f5c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f8a8e08b1e6b410b9a182595c69f7921": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_cb7f76795f9d47468b217c390c46f7a0",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_18d341df17db4c65a2aa25b229309748"
          }
        },
        "6600e51e4e5c40909206459f2349fa4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_52e3ed611d6344278687a1e5b0287438",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9920512/? [00:20&lt;00:00, 731805.29it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3c5ab01567f748beb64d83c6f5f358df"
          }
        },
        "cb7f76795f9d47468b217c390c46f7a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "18d341df17db4c65a2aa25b229309748": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "52e3ed611d6344278687a1e5b0287438": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3c5ab01567f748beb64d83c6f5f358df": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "749ebe7b1f00443b886a7e7fdfc988b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9cc66c0c7779474eb212f4bfafd6f7b1",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_97d3d043767847afbfd84cea80f03ca1",
              "IPY_MODEL_fe5460f798f447a4a8f8acb866429acc"
            ]
          }
        },
        "9cc66c0c7779474eb212f4bfafd6f7b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "97d3d043767847afbfd84cea80f03ca1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_69755afb32fc4f36959ecfaa2b99bbd8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7f754f2ec3b842418128a2c6f6a5fd7b"
          }
        },
        "fe5460f798f447a4a8f8acb866429acc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9eb4d8d5eabc4047b7e4bed35beba8ef",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 32768/? [00:01&lt;00:00, 20511.68it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3176016fc3374c8a84e1a3ddb26747f3"
          }
        },
        "69755afb32fc4f36959ecfaa2b99bbd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7f754f2ec3b842418128a2c6f6a5fd7b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9eb4d8d5eabc4047b7e4bed35beba8ef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3176016fc3374c8a84e1a3ddb26747f3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "159b76d1035d4392af0a184b824cf08a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ba5f54d4383749569e5413a13e139fa7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_e0aa94b802d64147ba05fd0b4ac92bfc",
              "IPY_MODEL_a8831685ebd94f88b64111d7e5182c76"
            ]
          }
        },
        "ba5f54d4383749569e5413a13e139fa7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e0aa94b802d64147ba05fd0b4ac92bfc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_864dc33bf152489194d57dd0c20d4c71",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_85faff9b3d98472985748260c3d9e58a"
          }
        },
        "a8831685ebd94f88b64111d7e5182c76": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8dee0633539a409f9d6f4e6269cc6aa8",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1654784/? [00:01&lt;00:00, 1285207.10it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_54d5e249630d4985b502e215255c85b9"
          }
        },
        "864dc33bf152489194d57dd0c20d4c71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "85faff9b3d98472985748260c3d9e58a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8dee0633539a409f9d6f4e6269cc6aa8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "54d5e249630d4985b502e215255c85b9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b772673a98674ce9840dcc3425cfa491": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_09a45b0c642e4a75ae28123af5639945",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6b8c120dcf56406a8428e5b74f05aa56",
              "IPY_MODEL_1a157a572ad4418693ca2c0510be732e"
            ]
          }
        },
        "09a45b0c642e4a75ae28123af5639945": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b8c120dcf56406a8428e5b74f05aa56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_f1d935bfff704604b532db2a8fe6e70a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_40b4b96cd5634477919f8a8666dd8e10"
          }
        },
        "1a157a572ad4418693ca2c0510be732e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_17b7509e0151462b9b28ba283f8277ca",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 8192/? [00:00&lt;00:00, 17985.88it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0f567e7ff3424899b459cc52da125600"
          }
        },
        "f1d935bfff704604b532db2a8fe6e70a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "40b4b96cd5634477919f8a8666dd8e10": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "17b7509e0151462b9b28ba283f8277ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0f567e7ff3424899b459cc52da125600": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D3Iw31CR86f1",
        "colab_type": "text"
      },
      "source": [
        "Deep Learning Models -- A collection of various deep learning architectures, models, and tips for TensorFlow and PyTorch in Jupyter Notebooks.\n",
        "- Author: Sebastian Raschka\n",
        "- GitHub Repository: https://github.com/rasbt/deeplearning-models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PnQMsUUm9Dkm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "21cc9d29-c0c7-4a63-ee5d-7211fcd68834"
      },
      "source": [
        "!pip install -q IPython\n",
        "!pip install -q ipykernel\n",
        "!pip install -q torch\n",
        "!pip install -q watermark\n",
        "!pip install -q matplotlib\n",
        "!pip install -q tensorwatch\n",
        "!pip install -q sklearn\n",
        "!pip install -q pandas\n",
        "!pip install -q pydot\n",
        "!pip install -q hiddenlayer\n",
        "!pip install -q graphviz\n",
        "!pip install -q torchvision\n",
        "!pip install -q torchtext"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 194kB 7.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 143kB 12.3MB/s \n",
            "\u001b[?25h  Building wheel for tensorwatch (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for pydotz (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "4m4wzdVy86f4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "outputId": "14dc3149-e1bf-413a-8fc2-28f04fcc93fd"
      },
      "source": [
        "%load_ext watermark\n",
        "%watermark -a 'Sebastian Raschka' -v -p torch"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sebastian Raschka \n",
            "\n",
            "CPython 3.6.9\n",
            "IPython 5.5.0\n",
            "\n",
            "torch 1.5.1+cu101\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BwvYTAgC86gB",
        "colab_type": "text"
      },
      "source": [
        "- Runs on CPU or GPU (if available)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uFOE15hI86gC",
        "colab_type": "text"
      },
      "source": [
        "# Model Zoo -- Multilayer Perceptron with BatchNorm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AOeUujRR86gE",
        "colab_type": "text"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XvgRQfy386gF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import time\n",
        "import numpy as np\n",
        "from torchvision import datasets\n",
        "from torchvision import transforms\n",
        "from torch.utils.data import DataLoader\n",
        "import torch.nn.functional as F\n",
        "import torch\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    torch.backends.cudnn.deterministic = True"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B0tB07Pz86gL",
        "colab_type": "text"
      },
      "source": [
        "## Settings and Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "DUrMN-vy86gM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 441,
          "referenced_widgets": [
            "a46b1e088aed4b04af22f27a0dfd431d",
            "3f29e77055514bfeb063b608d530f5c5",
            "f8a8e08b1e6b410b9a182595c69f7921",
            "6600e51e4e5c40909206459f2349fa4e",
            "cb7f76795f9d47468b217c390c46f7a0",
            "18d341df17db4c65a2aa25b229309748",
            "52e3ed611d6344278687a1e5b0287438",
            "3c5ab01567f748beb64d83c6f5f358df",
            "749ebe7b1f00443b886a7e7fdfc988b9",
            "9cc66c0c7779474eb212f4bfafd6f7b1",
            "97d3d043767847afbfd84cea80f03ca1",
            "fe5460f798f447a4a8f8acb866429acc",
            "69755afb32fc4f36959ecfaa2b99bbd8",
            "7f754f2ec3b842418128a2c6f6a5fd7b",
            "9eb4d8d5eabc4047b7e4bed35beba8ef",
            "3176016fc3374c8a84e1a3ddb26747f3",
            "159b76d1035d4392af0a184b824cf08a",
            "ba5f54d4383749569e5413a13e139fa7",
            "e0aa94b802d64147ba05fd0b4ac92bfc",
            "a8831685ebd94f88b64111d7e5182c76",
            "864dc33bf152489194d57dd0c20d4c71",
            "85faff9b3d98472985748260c3d9e58a",
            "8dee0633539a409f9d6f4e6269cc6aa8",
            "54d5e249630d4985b502e215255c85b9",
            "b772673a98674ce9840dcc3425cfa491",
            "09a45b0c642e4a75ae28123af5639945",
            "6b8c120dcf56406a8428e5b74f05aa56",
            "1a157a572ad4418693ca2c0510be732e",
            "f1d935bfff704604b532db2a8fe6e70a",
            "40b4b96cd5634477919f8a8666dd8e10",
            "17b7509e0151462b9b28ba283f8277ca",
            "0f567e7ff3424899b459cc52da125600"
          ]
        },
        "outputId": "6f290a18-4aa9-4caf-c865-198bd6c205d0"
      },
      "source": [
        "##########################\n",
        "### SETTINGS\n",
        "##########################\n",
        "\n",
        "# Device\n",
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "# Hyperparameters\n",
        "random_seed = 1\n",
        "learning_rate = 0.1\n",
        "num_epochs = 10\n",
        "batch_size = 64\n",
        "\n",
        "# Architecture\n",
        "num_features = 784\n",
        "num_hidden_1 = 128\n",
        "num_hidden_2 = 256\n",
        "num_classes = 10\n",
        "\n",
        "\n",
        "##########################\n",
        "### MNIST DATASET\n",
        "##########################\n",
        "\n",
        "# Note transforms.ToTensor() scales input images\n",
        "# to 0-1 range\n",
        "train_dataset = datasets.MNIST(root='data', \n",
        "                               train=True, \n",
        "                               transform=transforms.ToTensor(),\n",
        "                               download=True)\n",
        "\n",
        "test_dataset = datasets.MNIST(root='data', \n",
        "                              train=False, \n",
        "                              transform=transforms.ToTensor())\n",
        "\n",
        "\n",
        "train_loader = DataLoader(dataset=train_dataset, \n",
        "                          batch_size=batch_size, \n",
        "                          shuffle=True)\n",
        "\n",
        "test_loader = DataLoader(dataset=test_dataset, \n",
        "                         batch_size=batch_size, \n",
        "                         shuffle=False)\n",
        "\n",
        "# Checking the dataset\n",
        "for images, labels in train_loader:  \n",
        "    print('Image batch dimensions:', images.shape)\n",
        "    print('Image label dimensions:', labels.shape)\n",
        "    break"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to data/MNIST/raw/train-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a46b1e088aed4b04af22f27a0dfd431d",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/train-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "749ebe7b1f00443b886a7e7fdfc988b9",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/train-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "159b76d1035d4392af0a184b824cf08a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/t10k-images-idx3-ubyte.gz to data/MNIST/raw\n",
            "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b772673a98674ce9840dcc3425cfa491",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Extracting data/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/MNIST/raw\n",
            "Processing...\n",
            "Done!\n",
            "\n",
            "\n",
            "\n",
            "Image batch dimensions: torch.Size([64, 1, 28, 28])\n",
            "Image label dimensions: torch.Size([64])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/pytorch/torch/csrc/utils/tensor_numpy.cpp:141: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7wGJD55586gS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "##########################\n",
        "### MODEL\n",
        "##########################\n",
        "\n",
        "class MultilayerPerceptron(torch.nn.Module):\n",
        "\n",
        "    def __init__(self, num_features, num_classes):\n",
        "        super(MultilayerPerceptron, self).__init__()\n",
        "        \n",
        "        ### 1st hidden layer\n",
        "        self.linear_1 = torch.nn.Linear(num_features, num_hidden_1)\n",
        "        # The following to lones are not necessary, \n",
        "        # but used here to demonstrate how to access the weights\n",
        "        # and use a different weight initialization.\n",
        "        # By default, PyTorch uses Xavier/Glorot initialization, which\n",
        "        # should usually be preferred.\n",
        "        self.linear_1.weight.detach().normal_(0.0, 0.1)\n",
        "        self.linear_1.bias.detach().zero_()\n",
        "        self.linear_1_bn = torch.nn.BatchNorm1d(num_hidden_1)\n",
        "        \n",
        "        ### 2nd hidden layer\n",
        "        self.linear_2 = torch.nn.Linear(num_hidden_1, num_hidden_2)\n",
        "        self.linear_2.weight.detach().normal_(0.0, 0.1)\n",
        "        self.linear_2.bias.detach().zero_()\n",
        "        self.linear_2_bn = torch.nn.BatchNorm1d(num_hidden_2)\n",
        "        \n",
        "        ### Output layer\n",
        "        self.linear_out = torch.nn.Linear(num_hidden_2, num_classes)\n",
        "        self.linear_out.weight.detach().normal_(0.0, 0.1)\n",
        "        self.linear_out.bias.detach().zero_()\n",
        "        \n",
        "    def forward(self, x):\n",
        "        out = self.linear_1(x)\n",
        "        # note that batchnorm is in the classic\n",
        "        # sense placed before the activation\n",
        "        out = self.linear_1_bn(out)\n",
        "        out = F.relu(out)\n",
        "        \n",
        "        out = self.linear_2(out)\n",
        "        out = self.linear_2_bn(out)\n",
        "        out = F.relu(out)\n",
        "        \n",
        "        logits = self.linear_out(out)\n",
        "        probas = F.softmax(logits, dim=1)\n",
        "        return logits, probas\n",
        "\n",
        "    \n",
        "torch.manual_seed(random_seed)\n",
        "model = MultilayerPerceptron(num_features=num_features,\n",
        "                             num_classes=num_classes)\n",
        "\n",
        "model = model.to(device)\n",
        "\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "ijSfoRhz86gX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 185
        },
        "outputId": "ff74fe6e-9321-4260-aeee-8b6179ade932"
      },
      "source": [
        "import hiddenlayer as hl\n",
        "hl.build_graph(model, torch.zeros([64, 784]).to(device))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<hiddenlayer.graph.Graph at 0x7ff911ec7828>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: %3 Pages: 1 -->\n<svg width=\"1033pt\" height=\"108pt\"\n viewBox=\"0.00 0.00 1033.00 108.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(72 72)\">\n<title>%3</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-72,36 -72,-72 961,-72 961,36 -72,36\"/>\n<!-- /outputs/17 -->\n<g id=\"node1\" class=\"node\">\n<title>/outputs/17</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"54,-36 0,-36 0,0 54,0 54,-36\"/>\n<text text-anchor=\"start\" x=\"14\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear</text>\n</g>\n<!-- /outputs/18/19/20/21/22 -->\n<g id=\"node2\" class=\"node\">\n<title>/outputs/18/19/20/21/22</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"185,-36 122,-36 122,0 185,0 185,-36\"/>\n<text text-anchor=\"start\" x=\"130.5\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">BatchNorm</text>\n</g>\n<!-- /outputs/17&#45;&gt;/outputs/18/19/20/21/22 -->\n<g id=\"edge1\" class=\"edge\">\n<title>/outputs/17&#45;&gt;/outputs/18/19/20/21/22</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M54.1374,-18C70.921,-18 92.8065,-18 111.7644,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"111.9445,-21.5001 121.9445,-18 111.9444,-14.5001 111.9445,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"88\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128</text>\n</g>\n<!-- /outputs/23 -->\n<g id=\"node3\" class=\"node\">\n<title>/outputs/23</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"291,-36 237,-36 237,0 291,0 291,-36\"/>\n<text text-anchor=\"start\" x=\"254\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/18/19/20/21/22&#45;&gt;/outputs/23 -->\n<g id=\"edge2\" class=\"edge\">\n<title>/outputs/18/19/20/21/22&#45;&gt;/outputs/23</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M185.1649,-18C198.0473,-18 213.0358,-18 226.4633,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"226.8272,-21.5001 236.8271,-18 226.8271,-14.5001 226.8272,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"211\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">128</text>\n</g>\n<!-- /outputs/24 -->\n<g id=\"node4\" class=\"node\">\n<title>/outputs/24</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"413,-36 359,-36 359,0 413,0 413,-36\"/>\n<text text-anchor=\"start\" x=\"373\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear</text>\n</g>\n<!-- /outputs/23&#45;&gt;/outputs/24 -->\n<g id=\"edge3\" class=\"edge\">\n<title>/outputs/23&#45;&gt;/outputs/24</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M291.0758,-18C308.0553,-18 330.1767,-18 348.7924,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"349,-21.5001 358.9999,-18 348.9999,-14.5001 349,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"325\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x128</text>\n</g>\n<!-- /outputs/25/26/27/28/29 -->\n<g id=\"node5\" class=\"node\">\n<title>/outputs/25/26/27/28/29</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"544,-36 481,-36 481,0 544,0 544,-36\"/>\n<text text-anchor=\"start\" x=\"489.5\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">BatchNorm</text>\n</g>\n<!-- /outputs/24&#45;&gt;/outputs/25/26/27/28/29 -->\n<g id=\"edge4\" class=\"edge\">\n<title>/outputs/24&#45;&gt;/outputs/25/26/27/28/29</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M413.1374,-18C429.921,-18 451.8065,-18 470.7644,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"470.9445,-21.5001 480.9445,-18 470.9444,-14.5001 470.9445,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"447\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256</text>\n</g>\n<!-- /outputs/30 -->\n<g id=\"node6\" class=\"node\">\n<title>/outputs/30</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"650,-36 596,-36 596,0 650,0 650,-36\"/>\n<text text-anchor=\"start\" x=\"613\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Relu</text>\n</g>\n<!-- /outputs/25/26/27/28/29&#45;&gt;/outputs/30 -->\n<g id=\"edge5\" class=\"edge\">\n<title>/outputs/25/26/27/28/29&#45;&gt;/outputs/30</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M544.1649,-18C557.0473,-18 572.0358,-18 585.4633,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"585.8272,-21.5001 595.8271,-18 585.8271,-14.5001 585.8272,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"570\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">256</text>\n</g>\n<!-- /outputs/31 -->\n<g id=\"node7\" class=\"node\">\n<title>/outputs/31</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"772,-36 718,-36 718,0 772,0 772,-36\"/>\n<text text-anchor=\"start\" x=\"732\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Linear</text>\n</g>\n<!-- /outputs/30&#45;&gt;/outputs/31 -->\n<g id=\"edge6\" class=\"edge\">\n<title>/outputs/30&#45;&gt;/outputs/31</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M650.0758,-18C667.0553,-18 689.1767,-18 707.7924,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"708,-21.5001 717.9999,-18 707.9999,-14.5001 708,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"684\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x256</text>\n</g>\n<!-- /outputs/32 -->\n<g id=\"node8\" class=\"node\">\n<title>/outputs/32</title>\n<polygon fill=\"#e8e8e8\" stroke=\"#000000\" points=\"889,-36 835,-36 835,0 889,0 889,-36\"/>\n<text text-anchor=\"start\" x=\"845\" y=\"-15\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">Softmax</text>\n</g>\n<!-- /outputs/31&#45;&gt;/outputs/32 -->\n<g id=\"edge7\" class=\"edge\">\n<title>/outputs/31&#45;&gt;/outputs/32</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M772.1362,-18C787.6773,-18 807.4112,-18 824.4253,-18\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"824.6057,-21.5001 834.6056,-18 824.6056,-14.5001 824.6057,-21.5001\"/>\n<text text-anchor=\"middle\" x=\"803.5\" y=\"-21\" font-family=\"Times\" font-size=\"10.00\" fill=\"#000000\">64x10</text>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DkGseehv86gd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 439
        },
        "outputId": "e1cd0a40-7cb8-476b-f45c-922ebf12636e"
      },
      "source": [
        "import tensorwatch as tw\n",
        "tw.draw_model(model, [64, 784])"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorwatch.model_graph.hiddenlayer.pytorch_draw_model.DotWrapper at 0x7ff8ef948b38>"
            ],
            "image/svg+xml": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n -->\n<!-- Title: Net Pages: 1 -->\n<svg width=\"1302pt\" height=\"298pt\"\n viewBox=\"0.00 0.00 1301.70 298.48\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 294.4802)\">\n<title>Net</title>\n<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-294.4802 1297.7021,-294.4802 1297.7021,4 -4,4\"/>\n<!-- top_level_op -->\n<g id=\"node1\" class=\"node\">\n<title>top_level_op</title>\n<path fill=\"#6495ed\" stroke=\"#000000\" d=\"M21.1543,-251.9802C21.1543,-251.9802 89.1543,-251.9802 89.1543,-251.9802 95.1543,-251.9802 101.1543,-257.9802 101.1543,-263.9802 101.1543,-263.9802 101.1543,-277.9802 101.1543,-277.9802 101.1543,-283.9802 95.1543,-289.9802 89.1543,-289.9802 89.1543,-289.9802 21.1543,-289.9802 21.1543,-289.9802 15.1543,-289.9802 9.1543,-283.9802 9.1543,-277.9802 9.1543,-277.9802 9.1543,-263.9802 9.1543,-263.9802 9.1543,-257.9802 15.1543,-251.9802 21.1543,-251.9802\"/>\n<text text-anchor=\"middle\" x=\"55.1543\" y=\"-274.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op</text>\n<text text-anchor=\"middle\" x=\"55.1543\" y=\"-259.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">aten::t</text>\n</g>\n<!-- 51 -->\n<g id=\"node6\" class=\"node\">\n<title>51</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"55.1543\" cy=\"-188.6102\" rx=\"55.3091\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"55.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">100352</text>\n<text text-anchor=\"middle\" x=\"55.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(784, 128)</text>\n</g>\n<!-- top_level_op&#45;&gt;51 -->\n<g id=\"edge1\" class=\"edge\">\n<title>top_level_op&#45;&gt;51</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M55.1543,-251.8758C55.1543,-244.072 55.1543,-234.7666 55.1543,-225.7253\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"58.6544,-225.7238 55.1543,-215.7238 51.6544,-225.7239 58.6544,-225.7238\"/>\n</g>\n<!-- top_level_op_prim -->\n<g id=\"node2\" class=\"node\">\n<title>top_level_op_prim</title>\n<path fill=\"#6495ed\" stroke=\"#000000\" d=\"M810.1543,-251.9802C810.1543,-251.9802 994.1543,-251.9802 994.1543,-251.9802 1000.1543,-251.9802 1006.1543,-257.9802 1006.1543,-263.9802 1006.1543,-263.9802 1006.1543,-277.9802 1006.1543,-277.9802 1006.1543,-283.9802 1000.1543,-289.9802 994.1543,-289.9802 994.1543,-289.9802 810.1543,-289.9802 810.1543,-289.9802 804.1543,-289.9802 798.1543,-283.9802 798.1543,-277.9802 798.1543,-277.9802 798.1543,-263.9802 798.1543,-263.9802 798.1543,-257.9802 804.1543,-251.9802 810.1543,-251.9802\"/>\n<text text-anchor=\"middle\" x=\"902.1543\" y=\"-274.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op_prim::Constant_25</text>\n<text text-anchor=\"middle\" x=\"902.1543\" y=\"-259.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">prim::Constant</text>\n</g>\n<!-- top_level_op_aten -->\n<g id=\"node3\" class=\"node\">\n<title>top_level_op_aten</title>\n<path fill=\"#6495ed\" stroke=\"#000000\" d=\"M1036.6543,-251.9802C1036.6543,-251.9802 1211.6543,-251.9802 1211.6543,-251.9802 1217.6543,-251.9802 1223.6543,-257.9802 1223.6543,-263.9802 1223.6543,-263.9802 1223.6543,-277.9802 1223.6543,-277.9802 1223.6543,-283.9802 1217.6543,-289.9802 1211.6543,-289.9802 1211.6543,-289.9802 1036.6543,-289.9802 1036.6543,-289.9802 1030.6543,-289.9802 1024.6543,-283.9802 1024.6543,-277.9802 1024.6543,-277.9802 1024.6543,-263.9802 1024.6543,-263.9802 1024.6543,-257.9802 1030.6543,-251.9802 1036.6543,-251.9802\"/>\n<text text-anchor=\"middle\" x=\"1124.1543\" y=\"-274.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op_aten::softmax_26</text>\n<text text-anchor=\"middle\" x=\"1124.1543\" y=\"-259.7802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">aten::softmax</text>\n</g>\n<!-- logits -->\n<g id=\"node4\" class=\"node\">\n<title>logits</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"187.1543\" cy=\"-26.8701\" rx=\"44.5955\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"187.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">640</text>\n<text text-anchor=\"middle\" x=\"187.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(64, 10)</text>\n</g>\n<!-- top_level_op_aten: -->\n<g id=\"node32\" class=\"node\">\n<title>top_level_op_aten:</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"659.1543\" cy=\"-107.7401\" rx=\"82.5854\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"659.1543\" y=\"-104.0401\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op_aten:</text>\n</g>\n<!-- logits&#45;&gt;top_level_op_aten: -->\n<g id=\"edge50\" class=\"edge\">\n<title>logits&#45;&gt;top_level_op_aten::softmax_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M223.3505,-42.795C231.994,-46.9562 241.2375,-51.0297 250.1543,-53.7401 309.1249,-71.6656 471.4967,-89.3834 573.0787,-99.4043\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"573.0291,-102.9162 583.323,-100.4093 573.7125,-95.9496 573.0291,-102.9162\"/>\n</g>\n<!-- 77 -->\n<g id=\"node5\" class=\"node\">\n<title>77</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1249.1543\" cy=\"-26.8701\" rx=\"44.5955\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1249.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">640</text>\n<text text-anchor=\"middle\" x=\"1249.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(64, 10)</text>\n</g>\n<!-- 51&#45;&gt;top_level_op_aten: -->\n<g id=\"edge4\" class=\"edge\">\n<title>51&#45;&gt;top_level_op_aten::addmm_3</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M94.8113,-169.8187C102.7307,-166.6808 111.095,-163.7858 119.1543,-161.7401 202.0382,-140.7015 438.8803,-122.3486 569.6755,-113.4504\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"569.9363,-116.9408 579.6774,-112.7743 569.4641,-109.9568 569.9363,-116.9408\"/>\n</g>\n<!-- 79 -->\n<g id=\"node7\" class=\"node\">\n<title>79</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"155.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"155.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"155.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 79&#45;&gt;top_level_op_aten: -->\n<g id=\"edge5\" class=\"edge\">\n<title>79&#45;&gt;top_level_op_aten::addmm_3</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M175.3494,-170.6832C180.2348,-167.1847 185.6456,-163.9278 191.1543,-161.7401 257.655,-135.3305 451.8725,-119.7961 568.1598,-112.6079\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"568.7016,-116.0815 578.4702,-111.9798 568.2759,-109.0945 568.7016,-116.0815\"/>\n</g>\n<!-- 80 -->\n<g id=\"node8\" class=\"node\">\n<title>80</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"227.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"227.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"227.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 80&#45;&gt;top_level_op_aten: -->\n<g id=\"edge6\" class=\"edge\">\n<title>80&#45;&gt;top_level_op_aten::addmm_3</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M247.3833,-170.7674C252.267,-167.2646 257.6685,-163.9846 263.1543,-161.7401 317.2476,-139.6086 471.6342,-123.2204 570.8904,-114.6006\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"571.2514,-118.0826 580.9154,-113.74 570.6527,-111.1082 571.2514,-118.0826\"/>\n</g>\n<!-- input.2 -->\n<g id=\"node9\" class=\"node\">\n<title>input.2</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"300.1543\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"300.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8192</text>\n<text text-anchor=\"middle\" x=\"300.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(64, 128)</text>\n</g>\n<!-- input.2&#45;&gt;top_level_op_aten: -->\n<g id=\"edge12\" class=\"edge\">\n<title>input.2&#45;&gt;top_level_op_aten::batch_norm_8</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M340.4073,-43.0027C349.4501,-46.9676 359.0224,-50.888 368.1543,-53.7401 437.1552,-75.2907 517.9342,-89.2177 576.9094,-97.521\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"576.534,-101.0023 586.9194,-98.9059 577.4935,-94.0683 576.534,-101.0023\"/>\n</g>\n<!-- 81 -->\n<g id=\"node10\" class=\"node\">\n<title>81</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"299.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"299.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"299.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 81&#45;&gt;top_level_op_aten: -->\n<g id=\"edge13\" class=\"edge\">\n<title>81&#45;&gt;top_level_op_aten::batch_norm_8</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M319.8199,-170.611C324.5982,-167.2048 329.8458,-164.0044 335.1543,-161.7401 377.2825,-143.7711 494.1041,-127.0916 575.2986,-117.1579\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"576.006,-120.5979 585.5125,-115.9208 575.1642,-113.6487 576.006,-120.5979\"/>\n</g>\n<!-- 82 -->\n<g id=\"node11\" class=\"node\">\n<title>82</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"371.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"371.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"371.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 82&#45;&gt;top_level_op_aten: -->\n<g id=\"edge14\" class=\"edge\">\n<title>82&#45;&gt;top_level_op_aten::batch_norm_8</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M391.9009,-170.7949C396.6742,-167.3773 401.8992,-164.1257 407.1543,-161.7401 459.8494,-137.8188 523.6233,-124.2851 573.8305,-116.7488\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"574.611,-120.1724 584.0061,-115.2752 573.6076,-113.2447 574.611,-120.1724\"/>\n</g>\n<!-- 83 -->\n<g id=\"node12\" class=\"node\">\n<title>83</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"443.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"443.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"443.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 83&#45;&gt;top_level_op_aten: -->\n<g id=\"edge15\" class=\"edge\">\n<title>83&#45;&gt;top_level_op_aten::batch_norm_8</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M464.0526,-171.1121C468.8164,-167.6747 473.9993,-164.3349 479.1543,-161.7401 513.7684,-144.3172 555.0106,-131.4373 589.2792,-122.6353\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"590.3608,-125.9726 599.2106,-120.1475 588.6598,-119.1824 590.3608,-125.9726\"/>\n</g>\n<!-- 84 -->\n<g id=\"node13\" class=\"node\">\n<title>84</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"515.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"515.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"515.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 84&#45;&gt;top_level_op_aten: -->\n<g id=\"edge16\" class=\"edge\">\n<title>84&#45;&gt;top_level_op_aten::batch_norm_8</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M536.4199,-171.7697C541.1608,-168.2914 546.2415,-164.7686 551.1543,-161.7401 571.0272,-149.4898 594.0476,-137.6788 613.7165,-128.2431\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"615.2456,-131.3917 622.7835,-123.9465 612.248,-125.066 615.2456,-131.3917\"/>\n</g>\n<!-- out.1 -->\n<g id=\"node14\" class=\"node\">\n<title>out.1</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"418.1543\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"418.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8192</text>\n<text text-anchor=\"middle\" x=\"418.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(64, 128)</text>\n</g>\n<!-- out.1&#45;&gt;top_level_op_aten: -->\n<g id=\"edge18\" class=\"edge\">\n<title>out.1&#45;&gt;top_level_op_aten::relu_9</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M459.2755,-42.6146C468.1262,-46.4926 477.4044,-50.4676 486.1543,-53.7401 523.8041,-67.8211 566.8883,-80.7416 600.7105,-90.4115\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"599.8657,-93.8099 610.4416,-93.1781 601.78,-87.0768 599.8657,-93.8099\"/>\n</g>\n<!-- input.3 -->\n<g id=\"node15\" class=\"node\">\n<title>input.3</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"536.1543\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"536.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">8192</text>\n<text text-anchor=\"middle\" x=\"536.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(64, 128)</text>\n</g>\n<!-- input.3&#45;&gt;top_level_op_aten: -->\n<g id=\"edge23\" class=\"edge\">\n<title>input.3&#45;&gt;top_level_op_aten::addmm_13</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M571.6801,-46.3675C590.0237,-57.7377 612.0661,-72.1582 629.4943,-84.1636\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"627.9384,-87.3454 638.1436,-90.1925 631.9413,-81.6028 627.9384,-87.3454\"/>\n</g>\n<!-- 61 -->\n<g id=\"node16\" class=\"node\">\n<title>61</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"659.1543\" cy=\"-26.8701\" rx=\"55.3091\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"659.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">32768</text>\n<text text-anchor=\"middle\" x=\"659.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(128, 256)</text>\n</g>\n<!-- 61&#45;&gt;top_level_op_aten: -->\n<g id=\"edge24\" class=\"edge\">\n<title>61&#45;&gt;top_level_op_aten::addmm_13</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M665.5548,-53.7407C665.9943,-61.9682 666.0193,-71.0431 665.63,-79.3673\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"662.1356,-79.1657 664.9151,-89.3894 669.1178,-79.6638 662.1356,-79.1657\"/>\n</g>\n<!-- 85 -->\n<g id=\"node17\" class=\"node\">\n<title>85</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"587.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"587.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"587.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 85&#45;&gt;top_level_op_aten: -->\n<g id=\"edge25\" class=\"edge\">\n<title>85&#45;&gt;top_level_op_aten::addmm_13</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M605.323,-168.2032C614.8004,-157.5582 626.4837,-144.4357 636.5413,-133.1389\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"639.162,-135.4589 643.1975,-125.6627 633.9338,-130.8042 639.162,-135.4589\"/>\n</g>\n<!-- 86 -->\n<g id=\"node18\" class=\"node\">\n<title>86</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"659.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"659.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"659.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 86&#45;&gt;top_level_op_aten: -->\n<g id=\"edge26\" class=\"edge\">\n<title>86&#45;&gt;top_level_op_aten::addmm_13</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M659.1543,-161.6816C659.1543,-153.4111 659.1543,-144.2888 659.1543,-135.9358\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"662.6544,-135.8889 659.1543,-125.8889 655.6544,-135.889 662.6544,-135.8889\"/>\n</g>\n<!-- input.4 -->\n<g id=\"node19\" class=\"node\">\n<title>input.4</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"782.1543\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"782.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16384</text>\n<text text-anchor=\"middle\" x=\"782.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(64, 256)</text>\n</g>\n<!-- input.4&#45;&gt;top_level_op_aten: -->\n<g id=\"edge32\" class=\"edge\">\n<title>input.4&#45;&gt;top_level_op_aten::batch_norm_18</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M753.9619,-49.4338C737.5842,-60.7569 716.7897,-74.4206 698.8045,-85.6739\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"696.8907,-82.7423 690.2327,-90.9838 700.577,-88.693 696.8907,-82.7423\"/>\n</g>\n<!-- 87 -->\n<g id=\"node20\" class=\"node\">\n<title>87</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"731.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"731.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"731.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 87&#45;&gt;top_level_op_aten: -->\n<g id=\"edge33\" class=\"edge\">\n<title>87&#45;&gt;top_level_op_aten::batch_norm_18</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M712.9857,-168.2032C703.5083,-157.5582 691.825,-144.4357 681.7673,-133.1389\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"684.3748,-130.8042 675.1111,-125.6627 679.1467,-135.4589 684.3748,-130.8042\"/>\n</g>\n<!-- 88 -->\n<g id=\"node21\" class=\"node\">\n<title>88</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"803.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"803.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"803.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 88&#45;&gt;top_level_op_aten: -->\n<g id=\"edge34\" class=\"edge\">\n<title>88&#45;&gt;top_level_op_aten::batch_norm_18</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M781.8888,-171.7697C777.1478,-168.2914 772.0672,-164.7686 767.1543,-161.7401 747.2815,-149.4898 724.261,-137.6788 704.5921,-128.2431\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"706.0607,-125.066 695.5252,-123.9465 703.0631,-131.3917 706.0607,-125.066\"/>\n</g>\n<!-- 89 -->\n<g id=\"node22\" class=\"node\">\n<title>89</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"875.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"875.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"875.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 89&#45;&gt;top_level_op_aten: -->\n<g id=\"edge35\" class=\"edge\">\n<title>89&#45;&gt;top_level_op_aten::batch_norm_18</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M854.256,-171.1121C849.4922,-167.6747 844.3094,-164.3349 839.1543,-161.7401 804.5403,-144.3172 763.2981,-131.4373 729.0294,-122.6353\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"729.6489,-119.1824 719.0981,-120.1475 727.9479,-125.9726 729.6489,-119.1824\"/>\n</g>\n<!-- 90 -->\n<g id=\"node23\" class=\"node\">\n<title>90</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"947.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"947.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"947.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 90&#45;&gt;top_level_op_aten: -->\n<g id=\"edge36\" class=\"edge\">\n<title>90&#45;&gt;top_level_op_aten::batch_norm_18</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M926.4078,-170.7949C921.6345,-167.3773 916.4094,-164.1257 911.1543,-161.7401 858.4593,-137.8188 794.6853,-124.2851 744.4782,-116.7488\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"744.701,-113.2447 734.3026,-115.2752 743.6977,-120.1724 744.701,-113.2447\"/>\n</g>\n<!-- out -->\n<g id=\"node24\" class=\"node\">\n<title>out</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"900.1543\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"900.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16384</text>\n<text text-anchor=\"middle\" x=\"900.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(64, 256)</text>\n</g>\n<!-- out&#45;&gt;top_level_op_aten: -->\n<g id=\"edge38\" class=\"edge\">\n<title>out&#45;&gt;top_level_op_aten::relu_19</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M866.518,-47.0975C861.1067,-49.4533 855.5308,-51.7293 850.1543,-53.7401 810.0561,-68.7368 763.7935,-82.4173 726.7696,-92.2563\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"725.8604,-88.8763 717.0764,-94.8001 727.6373,-95.647 725.8604,-88.8763\"/>\n</g>\n<!-- input -->\n<g id=\"node25\" class=\"node\">\n<title>input</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1018.1543\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1018.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">16384</text>\n<text text-anchor=\"middle\" x=\"1018.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(64, 256)</text>\n</g>\n<!-- input&#45;&gt;top_level_op_aten: -->\n<g id=\"edge43\" class=\"edge\">\n<title>input&#45;&gt;top_level_op_aten::addmm_23</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M985.3552,-47.4071C979.7136,-49.768 973.8484,-51.9617 968.1543,-53.7401 894.7044,-76.6803 807.9084,-90.982 744.9739,-99.0575\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"744.1774,-95.6299 734.6915,-100.3487 745.0497,-102.5754 744.1774,-95.6299\"/>\n</g>\n<!-- 71 -->\n<g id=\"node26\" class=\"node\">\n<title>71</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1136.1543\" cy=\"-26.8701\" rx=\"50.41\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1136.1543\" y=\"-30.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">2560</text>\n<text text-anchor=\"middle\" x=\"1136.1543\" y=\"-15.6701\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(256, 10)</text>\n</g>\n<!-- 71&#45;&gt;top_level_op_aten: -->\n<g id=\"edge44\" class=\"edge\">\n<title>71&#45;&gt;top_level_op_aten::addmm_23</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1103.7549,-47.5432C1098.0031,-49.9131 1091.999,-52.0726 1086.1543,-53.7401 973.6282,-85.8437 838.5734,-98.9105 750.8532,-104.1967\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"750.4408,-100.7147 740.6606,-104.7887 750.8467,-107.7029 750.4408,-100.7147\"/>\n</g>\n<!-- 91 -->\n<g id=\"node27\" class=\"node\">\n<title>91</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1019.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1019.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1019.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 91&#45;&gt;top_level_op_aten: -->\n<g id=\"edge45\" class=\"edge\">\n<title>91&#45;&gt;top_level_op_aten::addmm_23</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M998.4888,-170.611C993.7104,-167.2048 988.4629,-164.0044 983.1543,-161.7401 941.0262,-143.7711 824.2046,-127.0916 743.01,-117.1579\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"743.1445,-113.6487 732.7961,-115.9208 742.3027,-120.5979 743.1445,-113.6487\"/>\n</g>\n<!-- 92 -->\n<g id=\"node28\" class=\"node\">\n<title>92</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1091.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1091.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1091.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 92&#45;&gt;top_level_op_aten: -->\n<g id=\"edge46\" class=\"edge\">\n<title>92&#45;&gt;top_level_op_aten::addmm_23</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1070.9254,-170.7674C1066.0416,-167.2646 1060.6402,-163.9846 1055.1543,-161.7401 1001.0611,-139.6086 846.6744,-123.2204 747.4183,-114.6006\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"747.656,-111.1082 737.3932,-113.74 747.0572,-118.0826 747.656,-111.1082\"/>\n</g>\n<!-- 93 -->\n<g id=\"node29\" class=\"node\">\n<title>93</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1163.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1163.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1163.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 93&#45;&gt;top_level_op_aten: -->\n<g id=\"edge51\" class=\"edge\">\n<title>93&#45;&gt;top_level_op_aten::softmax_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1142.9593,-170.6832C1138.0738,-167.1847 1132.6631,-163.9278 1127.1543,-161.7401 1060.6536,-135.3305 866.4361,-119.7961 750.1489,-112.6079\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"750.0328,-109.0945 739.8385,-111.9798 749.6071,-116.0815 750.0328,-109.0945\"/>\n</g>\n<!-- 76 -->\n<g id=\"node30\" class=\"node\">\n<title>76</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"1235.1543\" cy=\"-188.6102\" rx=\"27\" ry=\"26.7407\"/>\n<text text-anchor=\"middle\" x=\"1235.1543\" y=\"-192.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">0</text>\n<text text-anchor=\"middle\" x=\"1235.1543\" y=\"-177.4102\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">(0,)</text>\n</g>\n<!-- 76&#45;&gt;top_level_op_aten: -->\n<g id=\"edge52\" class=\"edge\">\n<title>76&#45;&gt;top_level_op_aten::softmax_26</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M1214.9837,-170.621C1210.097,-167.1258 1204.6795,-163.8859 1199.1543,-161.7401 1119.7631,-130.9081 883.1894,-116.6923 751.0363,-111.0005\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"751.064,-107.4987 740.925,-110.5729 750.7681,-114.4924 751.064,-107.4987\"/>\n</g>\n<!-- top_level_op_prim: -->\n<g id=\"node31\" class=\"node\">\n<title>top_level_op_prim:</title>\n<ellipse fill=\"none\" stroke=\"#000000\" cx=\"695.1543\" cy=\"-270.9802\" rx=\"85.2851\" ry=\"18\"/>\n<text text-anchor=\"middle\" x=\"695.1543\" y=\"-267.2802\" font-family=\"Times,serif\" font-size=\"14.00\" fill=\"#000000\">top_level_op_prim:</text>\n</g>\n<!-- top_level_op_prim:&#45;&gt;79 -->\n<g id=\"edge2\" class=\"edge\">\n<title>top_level_op_prim::Constant_1&#45;&gt;79</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M611.8442,-267.1119C488.978,-260.5129 265.6812,-244.8532 191.1543,-215.4802 188.7418,-214.5294 186.3486,-213.3722 184.0105,-212.0785\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"185.6561,-208.977 175.341,-206.5583 181.8964,-214.8816 185.6561,-208.977\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;80 -->\n<g id=\"edge3\" class=\"edge\">\n<title>top_level_op_prim::Constant_2&#45;&gt;80</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M613.9419,-265.4889C506.9243,-257.4075 325.014,-240.5097 263.1543,-215.4802 260.7505,-214.5076 258.3638,-213.3343 256.0303,-212.029\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"257.6795,-208.929 247.3705,-206.4845 253.905,-214.8242 257.6795,-208.929\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;81 -->\n<g id=\"edge8\" class=\"edge\">\n<title>top_level_op_prim::Constant_4&#45;&gt;81</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M617.7486,-263.478C527.6312,-253.973 384.7977,-236.2981 335.1543,-215.4802 332.7629,-214.4774 330.3854,-213.2818 328.0585,-211.9604\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"329.713,-208.8626 319.4126,-206.3821 325.918,-214.7446 329.713,-208.8626\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;82 -->\n<g id=\"edge9\" class=\"edge\">\n<title>top_level_op_prim::Constant_5&#45;&gt;82</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M614.3493,-265.1737C554.9634,-258.6158 473.4167,-244.7365 407.1543,-215.4802 404.8445,-214.4604 402.5432,-213.2688 400.2851,-211.9651\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"402.1593,-209.009 391.8638,-206.5081 398.3526,-214.8835 402.1593,-209.009\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;83 -->\n<g id=\"edge10\" class=\"edge\">\n<title>top_level_op_prim::Constant_6&#45;&gt;83</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M626.139,-260.3449C582.516,-251.927 525.7791,-237.7928 479.1543,-215.4802 476.8768,-214.3903 474.5995,-213.1463 472.3589,-211.8046\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"474.2483,-208.858 463.9756,-206.2651 470.3892,-214.6982 474.2483,-208.858\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;84 -->\n<g id=\"edge11\" class=\"edge\">\n<title>top_level_op_prim::Constant_7&#45;&gt;84</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M647.0006,-256.1167C618.3022,-246.366 581.6449,-232.3171 551.1543,-215.4802 548.944,-214.2597 546.717,-212.9181 544.5128,-211.5057\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"546.4356,-208.5803 536.2086,-205.8125 542.4774,-214.3538 546.4356,-208.5803\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;85 -->\n<g id=\"edge21\" class=\"edge\">\n<title>top_level_op_prim::Constant_11&#45;&gt;85</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M672.2503,-253.5117C656.1804,-241.2554 634.5372,-224.7484 617.0998,-211.4492\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"618.9943,-208.4923 608.9204,-205.2108 614.7492,-214.0582 618.9943,-208.4923\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;86 -->\n<g id=\"edge22\" class=\"edge\">\n<title>top_level_op_prim::Constant_12&#45;&gt;86</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M687.1648,-252.6996C683.295,-243.8454 678.5202,-232.9203 674.0159,-222.6142\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"677.201,-221.1621 669.9891,-213.4007 670.7868,-223.9655 677.201,-221.1621\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;87 -->\n<g id=\"edge28\" class=\"edge\">\n<title>top_level_op_prim::Constant_14&#45;&gt;87</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M703.1439,-252.6996C707.0136,-243.8454 711.7885,-232.9203 716.2928,-222.6142\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"719.5219,-223.9655 720.3196,-213.4007 713.1077,-221.1621 719.5219,-223.9655\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;88 -->\n<g id=\"edge29\" class=\"edge\">\n<title>top_level_op_prim::Constant_15&#45;&gt;88</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M718.0584,-253.5117C734.1282,-241.2554 755.7715,-224.7484 773.2088,-211.4492\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"775.5595,-214.0582 781.3883,-205.2108 771.3144,-208.4923 775.5595,-214.0582\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;89 -->\n<g id=\"edge30\" class=\"edge\">\n<title>top_level_op_prim::Constant_16&#45;&gt;89</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M743.3081,-256.1167C772.0064,-246.366 808.6638,-232.3171 839.1543,-215.4802 841.3647,-214.2597 843.5917,-212.9181 845.7958,-211.5057\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"847.8313,-214.3538 854.1,-205.8125 843.8731,-208.5803 847.8313,-214.3538\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;90 -->\n<g id=\"edge31\" class=\"edge\">\n<title>top_level_op_prim::Constant_17&#45;&gt;90</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M755.5532,-258.2152C809.4249,-246.3312 883.8692,-228.5377 911.1543,-215.4802 913.4319,-214.3903 915.7091,-213.1463 917.9497,-211.8046\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"919.9195,-214.6982 926.3331,-206.2651 916.0604,-208.858 919.9195,-214.6982\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;91 -->\n<g id=\"edge41\" class=\"edge\">\n<title>top_level_op_prim::Constant_21&#45;&gt;91</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M755.5432,-258.2568C766.7105,-255.9596 778.2788,-253.6171 789.1543,-251.4802 875.2032,-234.573 902.8099,-250.6235 983.1543,-215.4802 985.4676,-214.4684 987.7716,-213.2827 990.0316,-211.9834\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"991.9598,-214.9046 998.457,-206.5358 988.159,-209.0263 991.9598,-214.9046\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;92 -->\n<g id=\"edge42\" class=\"edge\">\n<title>top_level_op_prim::Constant_22&#45;&gt;92</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M754.4554,-258.0216C765.94,-255.6981 777.9012,-253.4065 789.1543,-251.4802 906.744,-231.3517 944.7156,-260.5998 1055.1543,-215.4802 1057.5549,-214.4995 1059.9392,-213.3202 1062.2709,-212.0106\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1064.4003,-214.8028 1070.927,-206.457 1060.6203,-208.9111 1064.4003,-214.8028\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;93 -->\n<g id=\"edge48\" class=\"edge\">\n<title>top_level_op_prim::Constant_24&#45;&gt;93</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M753.9104,-257.8648C765.5519,-255.5285 777.7103,-253.2722 789.1543,-251.4802 938.4075,-228.1089 986.5061,-270.6236 1127.1543,-215.4802 1129.5686,-214.5337 1131.963,-213.3797 1134.302,-212.0882\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1136.4139,-214.893 1142.9734,-206.5729 1132.6571,-208.9865 1136.4139,-214.893\"/>\n</g>\n<!-- top_level_op_prim:&#45;&gt;76 -->\n<g id=\"edge49\" class=\"edge\">\n<title>top_level_op_prim::Constant_25&#45;&gt;76</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M753.3825,-257.7961C765.1781,-255.4375 777.5293,-253.1904 789.1543,-251.4802 970.1298,-224.8568 1028.2411,-280.6692 1199.1543,-215.4802 1201.5772,-214.5561 1203.9781,-213.4187 1206.3217,-212.1391\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1208.4219,-214.9521 1215.0028,-206.6489 1204.6804,-209.0359 1208.4219,-214.9521\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;logits -->\n<g id=\"edge47\" class=\"edge\">\n<title>top_level_op_aten::addmm_23&#45;&gt;logits</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M581.7329,-101.3919C476.7217,-91.8615 295.2504,-72.9196 232.1543,-53.7401 229.8719,-53.0463 227.5681,-52.2632 225.2711,-51.4122\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"226.5191,-48.1412 215.9389,-47.586 223.8636,-54.618 226.5191,-48.1412\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;77 -->\n<g id=\"edge53\" class=\"edge\">\n<title>top_level_op_aten::softmax_26&#45;&gt;77</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M739.2365,-103.2158C867.1616,-95.3931 1111.6999,-77.834 1195.1543,-53.7401 1199.1723,-52.5801 1203.2508,-51.1281 1207.2733,-49.5034\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1208.7505,-52.6774 1216.5023,-45.4552 1205.9386,-46.267 1208.7505,-52.6774\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;input.2 -->\n<g id=\"edge7\" class=\"edge\">\n<title>top_level_op_aten::addmm_3&#45;&gt;input.2</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M583.6172,-100.3487C520.1055,-92.5475 427.6932,-77.9574 350.1543,-53.7401 347.5742,-52.9343 344.9589,-52.0432 342.3453,-51.0899\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"343.5411,-47.7994 332.9535,-47.4071 340.9856,-54.3162 343.5411,-47.7994\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;out.1 -->\n<g id=\"edge17\" class=\"edge\">\n<title>top_level_op_aten::batch_norm_8&#45;&gt;out.1</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M601.2323,-94.8001C562.8326,-84.8518 511.8358,-70.077 468.1543,-53.7401 465.8021,-52.8604 463.4117,-51.9299 461.0137,-50.9642\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"462.3662,-47.7361 451.7906,-47.0975 459.6597,-54.1917 462.3662,-47.7361\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;input.3 -->\n<g id=\"edge19\" class=\"edge\">\n<title>top_level_op_aten::relu_9&#45;&gt;input.3</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M628.0759,-90.9838C611.2037,-80.6466 590.3396,-67.1215 572.7099,-55.1596\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"574.5754,-52.1952 564.3468,-49.4338 570.6209,-57.9712 574.5754,-52.1952\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;61 -->\n<g id=\"edge20\" class=\"edge\">\n<title>top_level_op_aten::t_10&#45;&gt;61</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M653.3935,-89.3894C652.6378,-81.8196 652.3151,-72.7601 652.4254,-63.9182\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"655.9294,-63.8484 652.7539,-53.7407 648.933,-63.6225 655.9294,-63.8484\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;input.4 -->\n<g id=\"edge27\" class=\"edge\">\n<title>top_level_op_aten::addmm_13&#45;&gt;input.4</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M680.1651,-90.1925C696.0892,-78.943 718.3578,-64.1693 738.0728,-51.7218\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"740.0084,-54.6394 746.6286,-46.3675 736.2949,-48.7055 740.0084,-54.6394\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;out -->\n<g id=\"edge37\" class=\"edge\">\n<title>top_level_op_aten::batch_norm_18&#45;&gt;out</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M707.867,-93.1781C742.8615,-83.2811 790.7881,-69.211 832.1543,-53.7401 837.8965,-51.5926 843.8661,-49.1424 849.7931,-46.6157\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"851.2473,-49.8002 859.0332,-42.6146 848.4657,-43.3765 851.2473,-49.8002\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;input -->\n<g id=\"edge39\" class=\"edge\">\n<title>top_level_op_aten::relu_19&#45;&gt;input</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M731.3893,-98.9059C790.9887,-90.8035 877.1617,-76.5375 950.1543,-53.7401 956.1472,-51.8684 962.3296,-49.5366 968.4276,-47.0341\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"970.0703,-50.1389 977.9013,-43.0027 967.3293,-43.6978 970.0703,-50.1389\"/>\n</g>\n<!-- top_level_op_aten:&#45;&gt;71 -->\n<g id=\"edge40\" class=\"edge\">\n<title>top_level_op_aten::t_20&#45;&gt;71</title>\n<path fill=\"none\" stroke=\"#000000\" d=\"M740.2619,-104.0031C823.8101,-98.5943 956.9779,-85.4586 1068.1543,-53.7401 1074.1918,-52.0176 1080.3994,-49.7701 1086.5095,-47.308\"/>\n<polygon fill=\"#000000\" stroke=\"#000000\" points=\"1088.1393,-50.4192 1095.9923,-43.3071 1085.4182,-43.9697 1088.1393,-50.4192\"/>\n</g>\n</g>\n</svg>\n"
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "TM4NsEt586gj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "d2682b2f-7dfb-43b0-a40e-884303ce409d"
      },
      "source": [
        "def compute_accuracy(net, data_loader):\n",
        "    net.eval()\n",
        "    correct_pred, num_examples = 0, 0\n",
        "    with torch.no_grad():\n",
        "        for features, targets in data_loader:\n",
        "            features = features.view(-1, 28*28).to(device)\n",
        "            targets = targets.to(device)\n",
        "            logits, probas = net(features)\n",
        "            _, predicted_labels = torch.max(probas, 1)\n",
        "            num_examples += targets.size(0)\n",
        "            correct_pred += (predicted_labels == targets).sum()\n",
        "        return correct_pred.float()/num_examples * 100\n",
        "\n",
        "    \n",
        "start_time = time.time()\n",
        "for epoch in range(num_epochs):\n",
        "    model.train()\n",
        "    for batch_idx, (features, targets) in enumerate(train_loader):\n",
        "        \n",
        "        features = features.view(-1, 28*28).to(device)\n",
        "        targets = targets.to(device)\n",
        "\n",
        "        ### FORWARD AND BACK PROP\n",
        "        logits, probas = model(features)\n",
        "        cost = F.cross_entropy(logits, targets)\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        cost.backward()\n",
        "        \n",
        "        ### UPDATE MODEL PARAMETERS\n",
        "        optimizer.step()\n",
        "        \n",
        "        ### LOGGING\n",
        "        if not batch_idx % 50:\n",
        "            print ('Epoch: %03d/%03d | Batch %03d/%03d | Cost: %.4f' \n",
        "                   %(epoch+1, num_epochs, batch_idx, \n",
        "                     len(train_loader), cost))\n",
        "\n",
        "    print('Epoch: %03d/%03d training accuracy: %.2f%%' % (\n",
        "          epoch+1, num_epochs, \n",
        "          compute_accuracy(model, train_loader)))\n",
        "\n",
        "    print('Time elapsed: %.2f min' % ((time.time() - start_time)/60))\n",
        "    \n",
        "print('Total Training Time: %.2f min' % ((time.time() - start_time)/60))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 001/010 | Batch 000/938 | Cost: 2.5287\n",
            "Epoch: 001/010 | Batch 050/938 | Cost: 0.5931\n",
            "Epoch: 001/010 | Batch 100/938 | Cost: 0.4121\n",
            "Epoch: 001/010 | Batch 150/938 | Cost: 0.1960\n",
            "Epoch: 001/010 | Batch 200/938 | Cost: 0.1789\n",
            "Epoch: 001/010 | Batch 250/938 | Cost: 0.3476\n",
            "Epoch: 001/010 | Batch 300/938 | Cost: 0.2571\n",
            "Epoch: 001/010 | Batch 350/938 | Cost: 0.1699\n",
            "Epoch: 001/010 | Batch 400/938 | Cost: 0.4255\n",
            "Epoch: 001/010 | Batch 450/938 | Cost: 0.2882\n",
            "Epoch: 001/010 | Batch 500/938 | Cost: 0.1355\n",
            "Epoch: 001/010 | Batch 550/938 | Cost: 0.1459\n",
            "Epoch: 001/010 | Batch 600/938 | Cost: 0.2932\n",
            "Epoch: 001/010 | Batch 650/938 | Cost: 0.2294\n",
            "Epoch: 001/010 | Batch 700/938 | Cost: 0.1631\n",
            "Epoch: 001/010 | Batch 750/938 | Cost: 0.1825\n",
            "Epoch: 001/010 | Batch 800/938 | Cost: 0.1547\n",
            "Epoch: 001/010 | Batch 850/938 | Cost: 0.1627\n",
            "Epoch: 001/010 | Batch 900/938 | Cost: 0.1151\n",
            "Epoch: 001/010 training accuracy: 96.47%\n",
            "Time elapsed: 0.26 min\n",
            "Epoch: 002/010 | Batch 000/938 | Cost: 0.1339\n",
            "Epoch: 002/010 | Batch 050/938 | Cost: 0.1184\n",
            "Epoch: 002/010 | Batch 100/938 | Cost: 0.1136\n",
            "Epoch: 002/010 | Batch 150/938 | Cost: 0.0984\n",
            "Epoch: 002/010 | Batch 200/938 | Cost: 0.2281\n",
            "Epoch: 002/010 | Batch 250/938 | Cost: 0.0890\n",
            "Epoch: 002/010 | Batch 300/938 | Cost: 0.0761\n",
            "Epoch: 002/010 | Batch 350/938 | Cost: 0.0532\n",
            "Epoch: 002/010 | Batch 400/938 | Cost: 0.2134\n",
            "Epoch: 002/010 | Batch 450/938 | Cost: 0.2650\n",
            "Epoch: 002/010 | Batch 500/938 | Cost: 0.1687\n",
            "Epoch: 002/010 | Batch 550/938 | Cost: 0.1282\n",
            "Epoch: 002/010 | Batch 600/938 | Cost: 0.0389\n",
            "Epoch: 002/010 | Batch 650/938 | Cost: 0.0259\n",
            "Epoch: 002/010 | Batch 700/938 | Cost: 0.0672\n",
            "Epoch: 002/010 | Batch 750/938 | Cost: 0.2879\n",
            "Epoch: 002/010 | Batch 800/938 | Cost: 0.1094\n",
            "Epoch: 002/010 | Batch 850/938 | Cost: 0.0606\n",
            "Epoch: 002/010 | Batch 900/938 | Cost: 0.0757\n",
            "Epoch: 002/010 training accuracy: 97.83%\n",
            "Time elapsed: 0.52 min\n",
            "Epoch: 003/010 | Batch 000/938 | Cost: 0.0408\n",
            "Epoch: 003/010 | Batch 050/938 | Cost: 0.0985\n",
            "Epoch: 003/010 | Batch 100/938 | Cost: 0.1517\n",
            "Epoch: 003/010 | Batch 150/938 | Cost: 0.0612\n",
            "Epoch: 003/010 | Batch 200/938 | Cost: 0.0409\n",
            "Epoch: 003/010 | Batch 250/938 | Cost: 0.1053\n",
            "Epoch: 003/010 | Batch 300/938 | Cost: 0.1002\n",
            "Epoch: 003/010 | Batch 350/938 | Cost: 0.0625\n",
            "Epoch: 003/010 | Batch 400/938 | Cost: 0.0288\n",
            "Epoch: 003/010 | Batch 450/938 | Cost: 0.2416\n",
            "Epoch: 003/010 | Batch 500/938 | Cost: 0.0599\n",
            "Epoch: 003/010 | Batch 550/938 | Cost: 0.0755\n",
            "Epoch: 003/010 | Batch 600/938 | Cost: 0.1234\n",
            "Epoch: 003/010 | Batch 650/938 | Cost: 0.0493\n",
            "Epoch: 003/010 | Batch 700/938 | Cost: 0.0924\n",
            "Epoch: 003/010 | Batch 750/938 | Cost: 0.1138\n",
            "Epoch: 003/010 | Batch 800/938 | Cost: 0.0317\n",
            "Epoch: 003/010 | Batch 850/938 | Cost: 0.0654\n",
            "Epoch: 003/010 | Batch 900/938 | Cost: 0.0171\n",
            "Epoch: 003/010 training accuracy: 98.39%\n",
            "Time elapsed: 0.77 min\n",
            "Epoch: 004/010 | Batch 000/938 | Cost: 0.0795\n",
            "Epoch: 004/010 | Batch 050/938 | Cost: 0.0956\n",
            "Epoch: 004/010 | Batch 100/938 | Cost: 0.0948\n",
            "Epoch: 004/010 | Batch 150/938 | Cost: 0.0808\n",
            "Epoch: 004/010 | Batch 200/938 | Cost: 0.0755\n",
            "Epoch: 004/010 | Batch 250/938 | Cost: 0.0654\n",
            "Epoch: 004/010 | Batch 300/938 | Cost: 0.0263\n",
            "Epoch: 004/010 | Batch 350/938 | Cost: 0.0276\n",
            "Epoch: 004/010 | Batch 400/938 | Cost: 0.0705\n",
            "Epoch: 004/010 | Batch 450/938 | Cost: 0.0382\n",
            "Epoch: 004/010 | Batch 500/938 | Cost: 0.1288\n",
            "Epoch: 004/010 | Batch 550/938 | Cost: 0.1366\n",
            "Epoch: 004/010 | Batch 600/938 | Cost: 0.0684\n",
            "Epoch: 004/010 | Batch 650/938 | Cost: 0.0499\n",
            "Epoch: 004/010 | Batch 700/938 | Cost: 0.0354\n",
            "Epoch: 004/010 | Batch 750/938 | Cost: 0.0666\n",
            "Epoch: 004/010 | Batch 800/938 | Cost: 0.0498\n",
            "Epoch: 004/010 | Batch 850/938 | Cost: 0.0442\n",
            "Epoch: 004/010 | Batch 900/938 | Cost: 0.0735\n",
            "Epoch: 004/010 training accuracy: 98.82%\n",
            "Time elapsed: 1.03 min\n",
            "Epoch: 005/010 | Batch 000/938 | Cost: 0.0959\n",
            "Epoch: 005/010 | Batch 050/938 | Cost: 0.0373\n",
            "Epoch: 005/010 | Batch 100/938 | Cost: 0.1814\n",
            "Epoch: 005/010 | Batch 150/938 | Cost: 0.0608\n",
            "Epoch: 005/010 | Batch 200/938 | Cost: 0.0854\n",
            "Epoch: 005/010 | Batch 250/938 | Cost: 0.0550\n",
            "Epoch: 005/010 | Batch 300/938 | Cost: 0.0297\n",
            "Epoch: 005/010 | Batch 350/938 | Cost: 0.0462\n",
            "Epoch: 005/010 | Batch 400/938 | Cost: 0.0269\n",
            "Epoch: 005/010 | Batch 450/938 | Cost: 0.0219\n",
            "Epoch: 005/010 | Batch 500/938 | Cost: 0.1138\n",
            "Epoch: 005/010 | Batch 550/938 | Cost: 0.0650\n",
            "Epoch: 005/010 | Batch 600/938 | Cost: 0.1170\n",
            "Epoch: 005/010 | Batch 650/938 | Cost: 0.0831\n",
            "Epoch: 005/010 | Batch 700/938 | Cost: 0.0298\n",
            "Epoch: 005/010 | Batch 750/938 | Cost: 0.0635\n",
            "Epoch: 005/010 | Batch 800/938 | Cost: 0.0477\n",
            "Epoch: 005/010 | Batch 850/938 | Cost: 0.0297\n",
            "Epoch: 005/010 | Batch 900/938 | Cost: 0.0298\n",
            "Epoch: 005/010 training accuracy: 99.17%\n",
            "Time elapsed: 1.29 min\n",
            "Epoch: 006/010 | Batch 000/938 | Cost: 0.0290\n",
            "Epoch: 006/010 | Batch 050/938 | Cost: 0.0321\n",
            "Epoch: 006/010 | Batch 100/938 | Cost: 0.0267\n",
            "Epoch: 006/010 | Batch 150/938 | Cost: 0.1013\n",
            "Epoch: 006/010 | Batch 200/938 | Cost: 0.0114\n",
            "Epoch: 006/010 | Batch 250/938 | Cost: 0.0476\n",
            "Epoch: 006/010 | Batch 300/938 | Cost: 0.0156\n",
            "Epoch: 006/010 | Batch 350/938 | Cost: 0.0303\n",
            "Epoch: 006/010 | Batch 400/938 | Cost: 0.0126\n",
            "Epoch: 006/010 | Batch 450/938 | Cost: 0.0781\n",
            "Epoch: 006/010 | Batch 500/938 | Cost: 0.0529\n",
            "Epoch: 006/010 | Batch 550/938 | Cost: 0.0857\n",
            "Epoch: 006/010 | Batch 600/938 | Cost: 0.0330\n",
            "Epoch: 006/010 | Batch 650/938 | Cost: 0.0316\n",
            "Epoch: 006/010 | Batch 700/938 | Cost: 0.1251\n",
            "Epoch: 006/010 | Batch 750/938 | Cost: 0.0180\n",
            "Epoch: 006/010 | Batch 800/938 | Cost: 0.0310\n",
            "Epoch: 006/010 | Batch 850/938 | Cost: 0.0532\n",
            "Epoch: 006/010 | Batch 900/938 | Cost: 0.0302\n",
            "Epoch: 006/010 training accuracy: 99.33%\n",
            "Time elapsed: 1.55 min\n",
            "Epoch: 007/010 | Batch 000/938 | Cost: 0.0085\n",
            "Epoch: 007/010 | Batch 050/938 | Cost: 0.0409\n",
            "Epoch: 007/010 | Batch 100/938 | Cost: 0.0316\n",
            "Epoch: 007/010 | Batch 150/938 | Cost: 0.1429\n",
            "Epoch: 007/010 | Batch 200/938 | Cost: 0.0227\n",
            "Epoch: 007/010 | Batch 250/938 | Cost: 0.0618\n",
            "Epoch: 007/010 | Batch 300/938 | Cost: 0.0524\n",
            "Epoch: 007/010 | Batch 350/938 | Cost: 0.0308\n",
            "Epoch: 007/010 | Batch 400/938 | Cost: 0.0692\n",
            "Epoch: 007/010 | Batch 450/938 | Cost: 0.0888\n",
            "Epoch: 007/010 | Batch 500/938 | Cost: 0.0159\n",
            "Epoch: 007/010 | Batch 550/938 | Cost: 0.0423\n",
            "Epoch: 007/010 | Batch 600/938 | Cost: 0.0733\n",
            "Epoch: 007/010 | Batch 650/938 | Cost: 0.0314\n",
            "Epoch: 007/010 | Batch 700/938 | Cost: 0.1060\n",
            "Epoch: 007/010 | Batch 750/938 | Cost: 0.0216\n",
            "Epoch: 007/010 | Batch 800/938 | Cost: 0.0311\n",
            "Epoch: 007/010 | Batch 850/938 | Cost: 0.0263\n",
            "Epoch: 007/010 | Batch 900/938 | Cost: 0.0279\n",
            "Epoch: 007/010 training accuracy: 99.60%\n",
            "Time elapsed: 1.80 min\n",
            "Epoch: 008/010 | Batch 000/938 | Cost: 0.0463\n",
            "Epoch: 008/010 | Batch 050/938 | Cost: 0.0221\n",
            "Epoch: 008/010 | Batch 100/938 | Cost: 0.0209\n",
            "Epoch: 008/010 | Batch 150/938 | Cost: 0.0708\n",
            "Epoch: 008/010 | Batch 200/938 | Cost: 0.0085\n",
            "Epoch: 008/010 | Batch 250/938 | Cost: 0.0408\n",
            "Epoch: 008/010 | Batch 300/938 | Cost: 0.0651\n",
            "Epoch: 008/010 | Batch 350/938 | Cost: 0.0257\n",
            "Epoch: 008/010 | Batch 400/938 | Cost: 0.0489\n",
            "Epoch: 008/010 | Batch 450/938 | Cost: 0.0196\n",
            "Epoch: 008/010 | Batch 500/938 | Cost: 0.0245\n",
            "Epoch: 008/010 | Batch 550/938 | Cost: 0.0759\n",
            "Epoch: 008/010 | Batch 600/938 | Cost: 0.0293\n",
            "Epoch: 008/010 | Batch 650/938 | Cost: 0.0395\n",
            "Epoch: 008/010 | Batch 700/938 | Cost: 0.0414\n",
            "Epoch: 008/010 | Batch 750/938 | Cost: 0.0172\n",
            "Epoch: 008/010 | Batch 800/938 | Cost: 0.0501\n",
            "Epoch: 008/010 | Batch 850/938 | Cost: 0.0511\n",
            "Epoch: 008/010 | Batch 900/938 | Cost: 0.0165\n",
            "Epoch: 008/010 training accuracy: 99.73%\n",
            "Time elapsed: 2.06 min\n",
            "Epoch: 009/010 | Batch 000/938 | Cost: 0.0151\n",
            "Epoch: 009/010 | Batch 050/938 | Cost: 0.0164\n",
            "Epoch: 009/010 | Batch 100/938 | Cost: 0.0957\n",
            "Epoch: 009/010 | Batch 150/938 | Cost: 0.0108\n",
            "Epoch: 009/010 | Batch 200/938 | Cost: 0.0044\n",
            "Epoch: 009/010 | Batch 250/938 | Cost: 0.0159\n",
            "Epoch: 009/010 | Batch 300/938 | Cost: 0.0334\n",
            "Epoch: 009/010 | Batch 350/938 | Cost: 0.0381\n",
            "Epoch: 009/010 | Batch 400/938 | Cost: 0.0060\n",
            "Epoch: 009/010 | Batch 450/938 | Cost: 0.0386\n",
            "Epoch: 009/010 | Batch 500/938 | Cost: 0.0047\n",
            "Epoch: 009/010 | Batch 550/938 | Cost: 0.0795\n",
            "Epoch: 009/010 | Batch 600/938 | Cost: 0.0473\n",
            "Epoch: 009/010 | Batch 650/938 | Cost: 0.0295\n",
            "Epoch: 009/010 | Batch 700/938 | Cost: 0.0122\n",
            "Epoch: 009/010 | Batch 750/938 | Cost: 0.0395\n",
            "Epoch: 009/010 | Batch 800/938 | Cost: 0.0096\n",
            "Epoch: 009/010 | Batch 850/938 | Cost: 0.0619\n",
            "Epoch: 009/010 | Batch 900/938 | Cost: 0.0314\n",
            "Epoch: 009/010 training accuracy: 99.71%\n",
            "Time elapsed: 2.32 min\n",
            "Epoch: 010/010 | Batch 000/938 | Cost: 0.0044\n",
            "Epoch: 010/010 | Batch 050/938 | Cost: 0.0115\n",
            "Epoch: 010/010 | Batch 100/938 | Cost: 0.0524\n",
            "Epoch: 010/010 | Batch 150/938 | Cost: 0.0208\n",
            "Epoch: 010/010 | Batch 200/938 | Cost: 0.0091\n",
            "Epoch: 010/010 | Batch 250/938 | Cost: 0.0700\n",
            "Epoch: 010/010 | Batch 300/938 | Cost: 0.0357\n",
            "Epoch: 010/010 | Batch 350/938 | Cost: 0.0367\n",
            "Epoch: 010/010 | Batch 400/938 | Cost: 0.0145\n",
            "Epoch: 010/010 | Batch 450/938 | Cost: 0.0315\n",
            "Epoch: 010/010 | Batch 500/938 | Cost: 0.0041\n",
            "Epoch: 010/010 | Batch 550/938 | Cost: 0.0816\n",
            "Epoch: 010/010 | Batch 600/938 | Cost: 0.0166\n",
            "Epoch: 010/010 | Batch 650/938 | Cost: 0.0138\n",
            "Epoch: 010/010 | Batch 700/938 | Cost: 0.0152\n",
            "Epoch: 010/010 | Batch 750/938 | Cost: 0.0206\n",
            "Epoch: 010/010 | Batch 800/938 | Cost: 0.0138\n",
            "Epoch: 010/010 | Batch 850/938 | Cost: 0.0078\n",
            "Epoch: 010/010 | Batch 900/938 | Cost: 0.0179\n",
            "Epoch: 010/010 training accuracy: 99.82%\n",
            "Time elapsed: 2.57 min\n",
            "Total Training Time: 2.57 min\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "dBVjhyCx86go",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "ceef3180-741c-4849-99fc-0483ea63d669"
      },
      "source": [
        "print('Test accuracy: %.2f%%' % (compute_accuracy(model, test_loader)))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Test accuracy: 97.85%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "tags": [],
        "id": "Htqe2pIb86gs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "8e0a792e-e600-40c8-e7fb-88e0bd2bdd65"
      },
      "source": [
        "%watermark -iv"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "numpy 1.18.5\n",
            "torch 1.5.1+cu101\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}